{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from detail import Detail\n",
    "import numpy as np\n",
    "import skimage.io as io\n",
    "import matplotlib.pyplot as plt\n",
    "import os,sys\n",
    "import pylab\n",
    "import json\n",
    "import random\n",
    "from detail import mask as maskUtil\n",
    "pylab.rcParams['figure.figsize'] = (10.0, 8.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shutup(func):\n",
    "    def wrapper(*args, **kwargs):        \n",
    "        save_stdout = sys.stdout\n",
    "        sys.stdout = open(os.devnull, 'w')\n",
    "        res = func(*args, **kwargs)\n",
    "        sys.stdout = save_stdout\n",
    "        return res\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "annFile='/Users/zhishuaizhang/pascal_in_detail_data/trainval_merged.json' # annotations\n",
    "imgDir='/Users/zhishuaizhang/pascal_in_detail/VOCdevkit/VOC2010/JPEGImages' # jpeg images\n",
    "phase='trainval'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=8.42s)\n",
      "creating index...\n",
      "index created! 8.13s\n",
      "\n",
      "***info***\n",
      "description: This is the 3.0 version of the CVPR 2017 PASCAL in Detail dataset.\n",
      "version: 3.0\n",
      "year: 2017\n",
      "contributor: https://sites.google.com/view/pasd\n",
      "date_created: 2017-07-18\n"
     ]
    }
   ],
   "source": [
    "# initialize detail api for instance annotations\n",
    "details = Detail(annFile, imgDir, phase)\n",
    "\n",
    "print('\\n***info***')\n",
    "details.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from detail import instsegEval\n",
    "instsegeval = instsegEval.instsegEval(details)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test it on a toy dataset\n",
    "instsegeval.params.imgIds = [2008000512, 2008000514]\n",
    "instsegeval.params.catIds = [25, 360, 420, 284, 284, 284, 284, 284, 33, 232, 349, 440]\n",
    "# Create a detection result from groundtruth of 500 images\n",
    "# This cell will take 10 mins in my computer\n",
    "gts = [(i,j,maskUtil.encode(np.asfortranarray((details.getMask(i,cat=j).astype(np.uint8)==k).astype(np.uint8)))) \\\n",
    "    for i in instsegeval.params.imgIds \\\n",
    "        for j in [_['category_id'] for _ in details.getCats(imgs=i)] \\\n",
    "            for k in (set(np.unique(details.getMask(i,cat=j).astype(np.uint8)))-{0})]\n",
    "all_gt = []\n",
    "imgIds_ = []\n",
    "catIds_ = []\n",
    "[all_gt.append({'image_id':_[0],'category_id':_[1],'segmentation':{\"size\":_[2]['size'],\"counts\":_[2]['counts'].decode(\"utf-8\")},'score':100.}) or \\\n",
    "    imgIds_.append(_[0]) or catIds_.append(_[1]) for _ in gts]\n",
    "with open('/Users/zhishuaizhang/github_repo/detail-api/res/inst_seg_toy_one_wall_miss.json', 'w') as outfile:\n",
    "    json.dump(all_gt, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.769188165664673\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the toy_corrent dataset\n",
    "import time\n",
    "tic=time.time()\n",
    "instsegeval.loadSegRes('/Users/zhishuaizhang/github_repo/detail-api/res/inst_seg_toy_correct.json')\n",
    "instsegeval.evaluate()\n",
    "instsegeval.accumulate()\n",
    "toc=time.time()\n",
    "print(toc-tic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "print(instsegeval.ap[9,:,-1]) # AP for IOU>0.95, and unlimited detections\n",
    "print(instsegeval.ap[5,:,-1]) # AP for IOU>0.75, and unlimited detections\n",
    "print(instsegeval.ap[0,:,-1]) # AP for IOU>0.5, and unlimited detections\n",
    "## Work as expected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.058359146118164\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the toy_one_wall_miss dataset (There is only one wall instance)\n",
    "import time\n",
    "tic=time.time()\n",
    "instsegeval.loadSegRes('/Users/zhishuaizhang/github_repo/detail-api/res/inst_seg_toy_one_wall_miss.json')\n",
    "instsegeval.evaluate()\n",
    "instsegeval.accumulate()\n",
    "toc=time.time()\n",
    "print(toc-tic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.]\n"
     ]
    }
   ],
   "source": [
    "print(instsegeval.ap[9,:,-1]) # AP for IOU>0.95, and unlimited detections\n",
    "print(instsegeval.ap[5,:,-1]) # AP for IOU>0.75, and unlimited detections\n",
    "print(instsegeval.ap[0,:,-1]) # AP for IOU>0.5, and unlimited detections\n",
    "## Work as expected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a detection result from groundtruth of 500 images\n",
    "# This cell will take 10 mins in my computer\n",
    "gts = [(i,j,maskUtil.encode(np.asfortranarray((details.getMask(i,cat=j).astype(np.uint8)==k).astype(np.uint8)))) \\\n",
    "    for i in list(details.imgs.keys())[:500] \\\n",
    "        for j in [_['category_id'] for _ in details.getCats(imgs=i)] \\\n",
    "            for k in (set(np.unique(details.getMask(i,cat=j).astype(np.uint8)))-{0})]\n",
    "instsegeval = instsegEval.instsegEval(details)\n",
    "all_gt = []\n",
    "imgIds_ = []\n",
    "catIds_ = []\n",
    "[all_gt.append({'image_id':_[0],'category_id':_[1],'segmentation':{\"size\":_[2]['size'],\"counts\":_[2]['counts'].decode(\"utf-8\")},'score':100.}) or \\\n",
    "    imgIds_.append(_[0]) or catIds_.append(_[1]) for _ in gts]\n",
    "with open('/Users/zhishuaizhang/github_repo/detail-api/res/inst_seg_correct.json', 'w') as outfile:\n",
    "    json.dump(all_gt, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Simplify the image and category list to make debug / test easier\n",
    "instsegeval.params.imgIds = list(set(imgIds_))\n",
    "instsegeval.params.catIds = list(set(catIds_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Evaluate those 500 images\n",
    "# This cell will take 10 mins in my computer\n",
    "import time\n",
    "tic=time.time()\n",
    "instsegeval.loadSegRes('/Users/zhishuaizhang/github_repo/detail-api/res/inst_seg_correct.json')\n",
    "instsegeval.evaluate()\n",
    "instsegeval.accumulate()\n",
    "toc=time.time()\n",
    "print(toc-tic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.]\n",
      "[ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "print(instsegeval.ap[9,:,-1]) # AP for IOU>0.95, and unlimited detections\n",
    "print(instsegeval.ap[5,:,-1]) # AP for IOU>0.75, and unlimited detections\n",
    "print(instsegeval.ap[0,:,-1]) # AP for IOU>0.5, and unlimited detections\n",
    "## Work as expected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
